import json
from compressor import Compressor


def deal_line(**kwargs):
    content = kwargs['line']
    key_names = kwargs['key_names']

    content = content.replace('\\t', '')
    content = content.replace('\\n', '')
    content = content.replace('\\u', '')
    content = content.replace('\\"', '')
    content = content.replace('\\', '')
    content = content.replace('\\r\\n', ' ')

    dic = json.loads(content)

    data = {
        "body": dic[key_names[0]]
    }

    return json.dumps(data)


def run_pipeline(**kwargs):
    input_name = kwargs['input_name']
    output_name = kwargs['output_name']
    lines = []

    with open(input_name) as f:
        for line in f:
            try:
                content = deal_line(line=line, key_names=["text"])
            except Exception:
                continue

            lines.append(content)

    with open(output_name, 'w') as g:
        for line in lines:
            g.write(line + '\n')


if __name__ == "__main__":
    run_pipeline(
        input_name="/Users/wjunshen/Downloads/queries.jsonl",
        output_name="/Users/wjunshen/Downloads/queries.json",
    )
